{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|default_exp models.TSSequencerPlus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TSSequencerPlus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">This is a PyTorch implementation created by Ignacio Oguiza (oguiza@timeseriesAI.co) based on Sequencer: Deep LSTM for Image Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "from tsai.imports import *\n",
    "from tsai.models.utils import *\n",
    "from tsai.models.layers import *\n",
    "from typing import Callable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerEncoderLayer(nn.Module):\n",
    "    def __init__(self, d_model:int, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0., \n",
    "                 mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):\n",
    "        super().__init__()\n",
    "        self.bilstm = nn.LSTM(q_len, q_len, num_layers=1, bidirectional=True, bias=lstm_bias)\n",
    "        self.dropout = nn.Dropout(lstm_dropout)\n",
    "        self.fc = nn.Linear(2 * q_len, q_len)\n",
    "        self.lstm_norm = nn.LayerNorm(d_model)\n",
    "        self.pwff =  PositionwiseFeedForward(d_model, dropout=dropout, act=act, mlp_ratio=mlp_ratio)\n",
    "        self.ff_norm = nn.LayerNorm(d_model)\n",
    "        self.drop_path = DropPath(drop_path_rate) if drop_path_rate != 0 else nn.Identity()\n",
    "        self.pre_norm = pre_norm\n",
    "        self.transpose = Transpose(1,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.pre_norm:\n",
    "            x = self.drop_path(self.dropout(self.transpose(self.fc(self.bilstm(self.transpose(self.lstm_norm(x)))[0])))) + x\n",
    "            x = self.drop_path(self.pwff(self.ff_norm(x))) + x\n",
    "        else:\n",
    "            x = self.lstm_norm(self.drop_path(self.dropout(self.transpose(self.fc(self.bilstm(self.transpose(x))[0])))) + x)\n",
    "            x = self.ff_norm(self.drop_path(self.pwff(x)) + x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerEncoder(nn.Module):\n",
    "    def __init__(self, d_model, depth:int=6, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0., \n",
    "                 mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):\n",
    "        super().__init__()\n",
    "        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]\n",
    "        layers = []\n",
    "        for i in range(depth):\n",
    "            layer = _TSSequencerEncoderLayer(d_model, q_len=q_len, lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=dpr[i],\n",
    "                                      mlp_ratio=mlp_ratio, lstm_bias=lstm_bias, act=act, pre_norm=pre_norm)\n",
    "            layers.append(layer)\n",
    "        self.encoder = nn.Sequential(*layers)\n",
    "        self.norm = nn.LayerNorm(d_model) if pre_norm else nn.Identity()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.encoder(x)\n",
    "        x = self.norm(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerBackbone(Module):\n",
    "    def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str='gelu', \n",
    "                 lstm_bias:bool=True, lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, \n",
    "                 pre_norm:bool=False, use_token:bool=True,  use_pe:bool=True, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None, \n",
    "                 cat_padding_idxs:Optional[list]=None, cat_pos:Optional[list]=None, feature_extractor:Optional[Callable]=None, \n",
    "                 token_size:int=None, tokenizer:Optional[Callable]=None):\n",
    "\n",
    "        # Categorical embeddings\n",
    "        if n_cat_embeds is not None:\n",
    "            n_cat_embeds = listify(n_cat_embeds)\n",
    "            if cat_embed_dims is None:  \n",
    "                cat_embed_dims = [emb_sz_rule(s) for s in n_cat_embeds]\n",
    "            self.to_cat_embed = MultiEmbedding(c_in, n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos)\n",
    "            c_in, seq_len = output_size_calculator(self.to_cat_embed, c_in, seq_len)\n",
    "        else:\n",
    "            self.to_cat_embed = nn.Identity()\n",
    "            \n",
    "        # Sequence embedding\n",
    "        if token_size is not None:\n",
    "            self.tokenizer = SeqTokenizer(c_in, d_model, token_size)\n",
    "            c_in, seq_len = output_size_calculator(self.tokenizer, c_in, seq_len)\n",
    "        elif tokenizer is not None:\n",
    "            if isinstance(tokenizer, nn.Module):  self.tokenizer = tokenizer\n",
    "            else: self.tokenizer = tokenizer(c_in, d_model)\n",
    "            c_in, seq_len = output_size_calculator(self.tokenizer, c_in, seq_len)\n",
    "        else: \n",
    "            self.tokenizer = nn.Identity()\n",
    "\n",
    "        # Feature extractor\n",
    "        if feature_extractor is not None:\n",
    "            if isinstance(feature_extractor, nn.Module):  self.feature_extractor = feature_extractor\n",
    "            else: self.feature_extractor = feature_extractor(c_in, d_model)\n",
    "            c_in, seq_len = output_size_calculator(self.feature_extractor, c_in, seq_len)\n",
    "        else:\n",
    "            self.feature_extractor = nn.Identity()\n",
    "        \n",
    "        # Linear projection\n",
    "        if token_size is None and tokenizer is None and feature_extractor is None:\n",
    "            self.linear_proj = nn.Conv1d(c_in, d_model, 1)\n",
    "        else:\n",
    "            self.linear_proj = nn.Identity()\n",
    "            \n",
    "        self.transpose = Transpose(1,2)\n",
    "\n",
    "        # Position embedding & token\n",
    "        if use_pe:\n",
    "            self.pos_embed = nn.Parameter(torch.zeros(1, seq_len, d_model))\n",
    "        self.use_pe = use_pe\n",
    "        self.cls_token = nn.Parameter(torch.zeros(1, 1, d_model))\n",
    "        self.use_token = use_token\n",
    "        self.emb_dropout = nn.Dropout(dropout)\n",
    "\n",
    "        # Encoder\n",
    "        self.encoder = _TSSequencerEncoder(d_model, depth=depth, q_len=seq_len + use_token, lstm_bias=lstm_bias, \n",
    "                                         lstm_dropout=lstm_dropout, dropout=dropout,\n",
    "                                         mlp_ratio=mlp_ratio, drop_path_rate=drop_path_rate, act=act, pre_norm=pre_norm)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        # Categorical embeddings\n",
    "        x = self.to_cat_embed(x)\n",
    "        \n",
    "        # Sequence embedding\n",
    "        x = self.tokenizer(x)\n",
    "\n",
    "        # Feature extractor\n",
    "        x = self.feature_extractor(x)\n",
    "        \n",
    "        # Linear projection\n",
    "        x = self.linear_proj(x)\n",
    "        \n",
    "        # Position embedding & token\n",
    "        x = self.transpose(x)\n",
    "        if self.use_pe: \n",
    "            x = x + self.pos_embed\n",
    "        if self.use_token: # token is concatenated after position embedding so that embedding can be learned using self.supervised learning\n",
    "            x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)\n",
    "        x = self.emb_dropout(x)\n",
    "\n",
    "        # Encoder\n",
    "        x = self.encoder(x)\n",
    "        \n",
    "        # Output\n",
    "        x = x.transpose(1,2)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|exports\n",
    "class TSSequencerPlus(nn.Sequential):\n",
    "    r\"\"\"Time Series Sequencer model based on:\n",
    "\n",
    "    Tatsunami, Y., & Taki, M. (2022). Sequencer: Deep LSTM for Image Classification. arXiv preprint arXiv:2205.01972.\n",
    "    Official implementation: https://github.com/okojoalg/sequencer\n",
    "\n",
    "    Args:\n",
    "        c_in:               the number of features (aka variables, dimensions, channels) in the time series dataset.\n",
    "        c_out:              the number of target classes.\n",
    "        seq_len:            number of time steps in the time series.\n",
    "        d_model:            total dimension of the model (number of features created by the model).\n",
    "        depth:              number of blocks in the encoder.\n",
    "        act:                the activation function of positionwise feedforward layer.\n",
    "        lstm_dropout:       dropout rate applied to the lstm sublayer.\n",
    "        dropout:            dropout applied to to the embedded sequence steps after position embeddings have been added and \n",
    "                            to the mlp sublayer in the encoder.\n",
    "        drop_path_rate:     stochastic depth rate.\n",
    "        mlp_ratio:          ratio of mlp hidden dim to embedding dim.\n",
    "        lstm_bias:          determines whether bias is applied to the LSTM layer.\n",
    "        pre_norm:           if True normalization will be applied as the first step in the sublayers. Defaults to False.\n",
    "        use_token:          if True, the output will come from the transformed token. This is meant to be use in classification tasks.\n",
    "        use_pe:             flag to indicate if positional embedding is used.\n",
    "        n_cat_embeds:       list with the sizes of the dictionaries of embeddings (int).\n",
    "        cat_embed_dims:     list with the sizes of each embedding vector (int).\n",
    "        cat_padding_idxs:       If specified, the entries at cat_padding_idxs do not contribute to the gradient; therefore, the embedding vector at cat_padding_idxs\n",
    "                            are not updated during training. Use 0 for those categorical embeddings that may have #na# values. Otherwise, leave them as None.\n",
    "                            You can enter a combination for different embeddings (for example, [0, None, None]).\n",
    "        cat_pos:            list with the position of the categorical variables in the input.\n",
    "        token_size:         Size of the embedding function used to reduce the sequence length (similar to ViT's patch size)\n",
    "        tokenizer:          nn.Module or callable that will be used to reduce the sequence length\n",
    "        feature_extractor:  nn.Module or callable that will be used to preprocess the time series before \n",
    "                            the embedding step. It is useful to extract features or resample the time series.\n",
    "        flatten:            flag to indicate if the 3d logits will be flattened to 2d in the model's head if use_token is set to False. \n",
    "                            If use_token is False and flatten is False, the model will apply a pooling layer.\n",
    "        concat_pool:        if True the head begins with fastai's AdaptiveConcatPool2d if concat_pool=True; otherwise, it uses traditional average pooling. \n",
    "        fc_dropout:         dropout applied to the final fully connected layer.\n",
    "        use_bn:             flag that indicates if batchnorm will be applied to the head.\n",
    "        bias_init:          values used to initialized the output layer.\n",
    "        y_range:            range of possible y values (used in regression tasks).        \n",
    "        custom_head:        custom head that will be applied to the network. It must contain all kwargs (pass a partial function)\n",
    "        verbose:            flag to control verbosity of the model.\n",
    "\n",
    "    Input:\n",
    "        x: bs (batch size) x nvars (aka features, variables, dimensions, channels) x seq_len (aka time steps)\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=6, act:str='gelu',\n",
    "                 lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, lstm_bias:bool=True, \n",
    "                 pre_norm:bool=False, use_token:bool=False, use_pe:bool=True, \n",
    "                 cat_pos:Optional[list]=None, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None, cat_padding_idxs:Optional[list]=None,\n",
    "                 token_size:int=None, tokenizer:Optional[Callable]=None, feature_extractor:Optional[Callable]=None, \n",
    "                 flatten:bool=False, concat_pool:bool=True, fc_dropout:float=0., use_bn:bool=False, \n",
    "                 bias_init:Optional[Union[float, list]]=None, y_range:Optional[tuple]=None, custom_head:Optional[Callable]=None, verbose:bool=True,\n",
    "                 **kwargs):\n",
    "\n",
    "        if use_token and c_out == 1: \n",
    "            use_token = False\n",
    "            pv(\"use_token set to False as c_out == 1\", verbose)\n",
    "        backbone = _TSSequencerBackbone(c_in, seq_len, depth=depth, d_model=d_model, act=act,\n",
    "                                      lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=drop_path_rate, \n",
    "                                      pre_norm=pre_norm, mlp_ratio=mlp_ratio, use_pe=use_pe, use_token=use_token, \n",
    "                                      n_cat_embeds=n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos, \n",
    "                                      feature_extractor=feature_extractor, token_size=token_size, tokenizer=tokenizer)\n",
    "\n",
    "        self.head_nf = d_model\n",
    "        self.c_out = c_out\n",
    "        self.seq_len = seq_len\n",
    "\n",
    "        # Head\n",
    "        if custom_head:\n",
    "            if isinstance(custom_head, nn.Module): head = custom_head\n",
    "            else: head = custom_head(self.head_nf, c_out, seq_len, **kwargs)\n",
    "        else:\n",
    "            nf = d_model\n",
    "            layers = []\n",
    "            if use_token: \n",
    "                layers += [TokenLayer()]\n",
    "            elif flatten:\n",
    "                layers += [Reshape(-1)]\n",
    "                nf = nf * seq_len\n",
    "            else:\n",
    "                if concat_pool: nf *= 2\n",
    "                layers = [GACP1d(1) if concat_pool else GAP1d(1)]\n",
    "            if use_bn: layers += [nn.BatchNorm1d(nf)]\n",
    "            if fc_dropout: layers += [nn.Dropout(fc_dropout)]\n",
    "            \n",
    "            # Last layer\n",
    "            linear = nn.Linear(nf, c_out)\n",
    "            if bias_init is not None: \n",
    "                if isinstance(bias_init, float): nn.init.constant_(linear.bias, bias_init)\n",
    "                else: linear.bias = nn.Parameter(torch.as_tensor(bias_init, dtype=torch.float32))\n",
    "            layers += [linear]\n",
    "\n",
    "            if y_range: layers += [SigmoidRange(*y_range)]\n",
    "            head = nn.Sequential(*layers)\n",
    "        super().__init__(OrderedDict([('backbone', backbone), ('head', head)]))\n",
    "        \n",
    "        \n",
    "TSSequencer = TSSequencerPlus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, lstm_dropout=.1, dropout=.1, use_token=True)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, lstm_dropout=.1, dropout=.1, use_token=False)\n",
    "test_eq(model(xb).shape, (bs, c_out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = np.array([0.8, .2])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor(bias_init))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 1\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = 8.5\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor([bias_init]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = np.array([0.8, .2])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor(bias_init))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature extractor\n",
    "\n",
    "It's a known fact that transformers cannot be directly applied to long sequences. To avoid this, we have included a way to subsample the sequence to generate a more manageable input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsai.data.validation import get_splits\n",
    "from tsai.data.core import get_ts_dls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABZcAAABoCAYAAACNDM73AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAa00lEQVR4nO3deXRU5eHG8edmshKyEAwhQbI0BkR2QkzZFA5LgpUKLohiG7AFLUuKKfgjHs2CCArKSdkEpCU9VSpqRamySiWAoqxBEQSExMSChioQA0KWmd8flpGRQHJJZoaE7+ecOWfmnXvv+8zoFfqct+8YNpvNJgAAAAAAAAAATPBwdwAAAAAAAAAAQMNDuQwAAAAAAAAAMI1yGQAAAAAAAABgGuUyAAAAAAAAAMA0ymUAAAAAAAAAgGmUywAAAAAAAAAA0yiXAQAAAAAAAACmUS4DAAAAAAAAAEyjXAYAAAAAAAAAmEa5DAAA4CS5ubkyDEOFhYX2sb59+6pv3771PldWVpYMw3AYi46O1qhRo+p9rp8rLCyUYRjKzc21j40aNUpNmzZ1+twXGIahrKwsl80HAAAAgHIZAADA7tNPP9W9996rqKgo+fr6qlWrVho4cKDmzZvntDmPHTumrKws5efnO20OM1avXn3NlrTXcjYAAADgeuTp7gAAAADXgg8//FD9+vVTZGSkxowZo5YtW6q4uFgfffSR/vznP2vixIn1Ms/69esdXh87dkzZ2dmKjo5Wly5d6mWOCw4ePCgPD3NrCVavXq0FCxaYKnGjoqL0ww8/yMvLy2RCc66U7YcffpCnJ3+1BQAAAFyJv4EDAABIeuaZZxQUFKQdO3YoODjY4b2SkpJ6m8fb27verlUTHx8fp16/srJSVqtV3t7e8vX1depcNXH3/AAAAMD1iG0xAAAAJB05ckTt27e/pFiWpBYtWji8NgxDEyZM0CuvvKK2bdvK19dX8fHx2rx5c43zXLzn8qZNm5SQkCBJGj16tAzDuGTv4ups3bpVCQkJ8vX1VWxsrBYvXlztcT/fc7miokLZ2dmKi4uTr6+vmjdvrt69e2vDhg2SftwnecGCBfbPeOEh/bSv8vPPP6+cnBzFxsbKx8dH+/fvr3bP5QuOHj2qpKQk+fv7KyIiQtOmTZPNZrO/v2nTJhmGoU2bNjmc9/NrXinbhbGfr2jes2ePBg8erMDAQDVt2lT9+/fXRx995HDMhX2xP/jgA6WlpSk0NFT+/v4aNmyYTpw4Uf0/AAAAAACSWLkMAAAg6cetHbZt26Z9+/apQ4cONR6fl5enFStWKDU1VT4+Plq4cKGSk5O1ffv2Wp0vSe3atdO0adOUkZGhsWPHqk+fPpKknj17XvacTz/9VIMGDVJoaKiysrJUWVmpzMxMhYWF1ThfVlaWZs6cqd///ve69dZbVVpaqp07d2r37t0aOHCgHnnkER07dkwbNmzQ3//+92qvsWzZMp07d05jx46Vj4+PQkJCZLVaqz22qqpKycnJ+uUvf6lZs2Zp7dq1yszMVGVlpaZNm1aLb+gntcl2sc8++0x9+vRRYGCgHn/8cXl5eWnx4sXq27ev8vLylJiY6HD8xIkT1axZM2VmZqqwsFA5OTmaMGGCVqxYYSonAAAAcD2hXAYAAJA0efJkDR48WF26dNGtt96qPn36qH///urXr1+1ewnv27dPO3fuVHx8vCRpxIgRatu2rTIyMvTmm2/Was6wsDANHjxYGRkZ6tGjhx566KEaz8nIyJDNZtOWLVsUGRkpSbrnnnvUsWPHGs999913dccdd2jJkiXVvt+jRw+1adNGGzZsuGyWr776Sl988YVCQ0PtY4WFhdUee+7cOSUnJ2vu3LmSpHHjxmnIkCF67rnnlJqaqhtuuKHGzGayXezJJ59URUWFtm7dql/84heSpN/+9rdq27atHn/8ceXl5Tkc37x5c61fv96+GtpqtWru3Lk6ffq0goKCap0TAAAAuJ6wLQYAAICkgQMHatu2bfr1r3+tvXv3atasWUpKSlKrVq20atWqS47v0aOHvViWpMjISN11111at26dqqqqnJKxqqpK69at09ChQ+3FsvTjCuikpKQazw8ODtZnn32mw4cPX3WGe+65x6FYrsmECRPszy9sJ1JeXq733nvvqjPUpKqqSuvXr9fQoUPtxbIkhYeH68EHH9TWrVtVWlrqcM7YsWMdttno06ePqqqq9OWXXzotJwAAANDQUS4DAAD8T0JCgt58802dPHlS27dvV3p6ur7//nvde++92r9/v8OxcXFxl5zfpk0bnT171ml79Z44cUI//PBDtXO3bdu2xvOnTZumU6dOqU2bNurYsaOmTJmiTz75xFSGmJiYWh/r4eHhUO5KP35H0uVXO9eHEydO6OzZs9V+J+3atZPValVxcbHD+MVlvSQ1a9ZMknTy5Emn5QQAAAAaOsplAACAn/H29lZCQoJmzJihF198URUVFXr99dfdHavObrvtNh05ckR//etf1aFDBy1dulTdunXT0qVLa30NPz+/es108Wrhizlr9fflWCyWascv/vFBAAAAAI4olwEAAK6ge/fukqTjx487jFe3tcShQ4fUpEkTU9tGXK5crU5oaKj8/PyqnfvgwYO1ukZISIhGjx6tf/zjHyouLlanTp2UlZV1VXlqYrVadfToUYexQ4cOSZKio6Ml/bRC+NSpUw7HVbcdRW2zhYaGqkmTJtV+J59//rk8PDzUunXrWl0LAAAAwOVRLgMAAEh6//33q12lunr1akmXbjuxbds27d692/66uLhYb7/9tgYNGnTZVbDV8ff3l3RpuVodi8WipKQkvfXWWyoqKrKPHzhwQOvWravx/G+//dbhddOmTXXTTTfp/PnzV5WnNubPn29/brPZNH/+fHl5eal///6SpKioKFksFm3evNnhvIULF15yrdpms1gsGjRokN5++22H7Te++eYbLV++XL1791ZgYOBVfiIAAAAAF3i6OwAAAMC1YOLEiTp79qyGDRumm2++WeXl5frwww+1YsUKRUdHa/To0Q7Hd+jQQUlJSUpNTZWPj4+9DM3OzjY1b2xsrIKDg7Vo0SIFBATI399fiYmJl93bODs7W2vXrlWfPn00btw4VVZWat68eWrfvn2N+yffcsst6tu3r+Lj4xUSEqKdO3fqjTfecPjRvQs/UpiamqqkpCRZLBaNGDHC1Ge6wNfXV2vXrlVKSooSExO1Zs0avfvuu3riiSfsq7uDgoJ03333ad68eTIMQ7GxsXrnnXdUUlJyyfXMZJs+fbo2bNig3r17a9y4cfL09NTixYt1/vx5zZo166o+DwAAAABHlMsAAACSnn/+eb3++utavXq1lixZovLyckVGRmrcuHF68sknFRwc7HD87bffrh49eig7O1tFRUW65ZZblJubq06dOpma18vLS3/729+Unp6uRx99VJWVlVq2bNlly+VOnTpp3bp1SktLU0ZGhm688UZlZ2fr+PHjNZbLqampWrVqldavX6/z588rKipK06dP15QpU+zH3H333Zo4caJeffVVvfzyy7LZbFddLlssFq1du1Z/+MMfNGXKFAUEBCgzM1MZGRkOx82bN08VFRVatGiRfHx8NHz4cM2ePVsdOnRwOM5Mtvbt22vLli1KT0/XzJkzZbValZiYqJdfflmJiYlX9XkAAAAAODJs/EoJAACAKYZhaPz48Q5bPgAAAADA9YY9lwEAAAAAAAAAplEuAwAAAAAAAABMo1wGAAAAAAAAAJjGD/oBAACYxE9WAAAAAAArlwEAAAAAAAAAV4FyGQAAAAAAAABgmsu3xbBarTp27JgCAgJkGIarpwcAAAAAAAAaNJvNpu+//14RERHy8GDtKNzH5eXysWPH1Lp1a1dPCwAAAAAAADQqxcXFuvHGG90dA9cxl5fLAQEB/3tWLCnQ1dMDcLLOebe5OwIAJ9l7+2Z3RwAAAAAgSSqV1Pqing1wD5eXyz9thREoymWg8bE0tbg7AgCn4c9tAAAA4FrClrNwNzZlAQAAAAAAAACYRrkMAAAAAAAAADCNchkAAAAAAAAAYJrL91wGAAAAAAAAAGeoqqpSRUWFu2M0WBaLRZ6enrXez5tyGQAAAAAAAECDV1ZWpq+++ko2m83dURq0Jk2aKDw8XN7e3jUeS7kMAAAAAAAAoEGrqqrSV199pSZNmig0NLTWK2/xE5vNpvLycp04cUIFBQWKi4uTh8eVd1WmXAYAAAAAAADQoFVUVMhmsyk0NFR+fn7ujtNg+fn5ycvLS19++aXKy8vl6+t7xeP5QT8AAAAAAAAAjQIrluuuptXKDsc6MQcAAAAAAAAAoJGiXAYAAAAAAAAAmEa5DAAAAAAAAACNRHR0tHJyclwyF+UyAAAAAAAAgEbJMFz7MJfNuOIjKyvrqj7zjh07NHbs2Ks61yzT5fLmzZs1ZMgQRUREyDAMvfXWW06IBQAAAAAAAACN1/Hjx+2PnJwcBQYGOoxNnjzZfqzNZlNlZWWtrhsaGqomTZo4K7YD0+XymTNn1LlzZy1YsMAZeQAAAAAAAACg0WvZsqX9ERQUJMMw7K8///xzBQQEaM2aNYqPj5ePj4+2bt2qI0eO6K677lJYWJiaNm2qhIQEvffeew7X/fm2GIZhaOnSpRo2bJiaNGmiuLg4rVq1ql4+g+lyefDgwZo+fbqGDRtWLwEAAAAAAAAAAJeaOnWqnn32WR04cECdOnVSWVmZ7rjjDm3cuFF79uxRcnKyhgwZoqKioiteJzs7W8OHD9cnn3yiO+64QyNHjtR3331X53xO33P5/PnzKi0tdXgAAAAAAAAAAK5s2rRpGjhwoGJjYxUSEqLOnTvrkUceUYcOHRQXF6enn35asbGxNa5EHjVqlB544AHddNNNmjFjhsrKyrR9+/Y653N6uTxz5kwFBQXZH61bt3b2lAAAAAAAAADQ4HXv3t3hdVlZmSZPnqx27dopODhYTZs21YEDB2pcudypUyf7c39/fwUGBqqkpKTO+ZxeLqenp+v06dP2R3FxsbOnBAAAAAAAAIAGz9/f3+H15MmTtXLlSs2YMUNbtmxRfn6+OnbsqPLy8itex8vLy+G1YRiyWq11zudZ5yvUwMfHRz4+Ps6eBgAAAAAAAAAatQ8++ECjRo2y/x5eWVmZCgsL3ZbH6SuXAQAAAAAAAAB1FxcXpzfffFP5+fnau3evHnzwwXpZgXy1TK9cLisr0xdffGF/XVBQoPz8fIWEhCgyMrJewwEAAAAAAADA1bLZ3J2gfs2ZM0cPP/ywevbsqRtuuEH/93//p9LSUrflMWw2c1/xpk2b1K9fv0vGU1JSlJubW+P5paWlCgoKknRaUqCZqQE0AN12xbs7AgAn2R2/y90RAAAAAEiSSiUF6fTp0woMpF+TpHPnzqmgoEAxMTHy9fV1d5wGzcx3aXrlct++fWWyjwYAAAAAAAAANDLsuQwAAAAAAAAAMI1yGQAAAAAAAABgGuUyAAAAAAAAAMA0ymUAAAAAAAAAgGmUywAAAAAAAAAA0yiXAQAAAAAAAACmUS4DAAAAAAAAAEyjXAYAAAAAAAAAmEa5DAAAAAAAAAAwzdPdAQAAAAAAAADAGeJ3x7t0vl3ddtX6WMMwrvh+ZmamsrKyriqHYRhauXKlhg4delXn1xblMgAAAAAAAAC42PHjx+3PV6xYoYyMDB08eNA+1rRpU3fEMsXl5bLNZvvfs1JXTw3ABarKqtwdAYDT8Gc3AAAAcG348e/mP/VsaIhatmxpfx4UFCTDMBzGli5dqhdeeEEFBQWKjo5Wamqqxo0bJ0kqLy9XWlqa/vnPf+rkyZMKCwvTo48+qvT0dEVHR0uShg0bJkmKiopSYWGhUz6Dy8vlb7/99n/PWrt6agAusPd2dycA4DxB7g4AAAAA4CLffvutgoL4e3pj9MorrygjI0Pz589X165dtWfPHo0ZM0b+/v5KSUnR3LlztWrVKr322muKjIxUcXGxiouLJUk7duxQixYttGzZMiUnJ8tisTgtp8vL5ZCQEElSUVER//IDjUxpaalat26t4uJiBQYGujsOgHrE/Q00XtzfQOPF/Q00XqdPn1ZkZKS9Z0Pjk5mZqRdeeEF33323JCkmJkb79+/X4sWLlZKSoqKiIsXFxal3794yDENRUVH2c0NDQyVJwcHBDiuhncHl5bKHh4ekH5d684cb0DgFBgZyfwONFPc30HhxfwONF/c30Hhd6NnQuJw5c0ZHjhzR7373O40ZM8Y+XllZaV+sO2rUKA0cOFBt27ZVcnKy7rzzTg0aNMjlWflBPwAAAAAAAAC4RpSVlUmSXnrpJSUmJjq8d2GLi27duqmgoEBr1qzRe++9p+HDh2vAgAF64403XJqVchkAAAAAAAAArhFhYWGKiIjQ0aNHNXLkyMseFxgYqPvvv1/333+/7r33XiUnJ+u7775TSEiIvLy8VFVV5fSsLi+XfXx8lJmZKR8fH1dPDcDJuL+Bxov7G2i8uL+Bxov7G2i8uL8bv+zsbKWmpiooKEjJyck6f/68du7cqZMnTyotLU1z5sxReHi4unbtKg8PD73++utq2bKlgoODJUnR0dHauHGjevXqJR8fHzVr1swpOQ2bzWZzypUBAAAAAAAAwAXOnTungoICxcTEyNfX191xTMvNzdWkSZN06tQp+9jy5cs1e/Zs7d+/X/7+/urYsaMmTZqkYcOG6aWXXtLChQt1+PBhWSwWJSQkaPbs2eratask6V//+pfS0tJUWFioVq1aqbCwsNZZzHyXlMsAAAAAAAAAGrSGXi5fS8x8l/ykJAAAAAAAAADANMplAAAAAAAAAIBplMsAAAAAAAAAANMolwEAAAAAAAAAprm0XF6wYIGio6Pl6+urxMREbd++3ZXTA3CCmTNnKiEhQQEBAWrRooWGDh2qgwcPujsWACd49tlnZRiGJk2a5O4oAOrBf/7zHz300ENq3ry5/Pz81LFjR+3cudPdsQDUUVVVlZ566inFxMTIz89PsbGxevrpp2Wz2dwdDYBJmzdv1pAhQxQRESHDMPTWW285vG+z2ZSRkaHw8HD5+flpwIABOnz4sHvCXkP4713dmfkOXVYur1ixQmlpacrMzNTu3bvVuXNnJSUlqaSkxFURADhBXl6exo8fr48++kgbNmxQRUWFBg0apDNnzrg7GoB6tGPHDi1evFidOnVydxQA9eDkyZPq1auXvLy8tGbNGu3fv18vvPCCmjVr5u5oAOroueee04svvqj58+frwIEDeu655zRr1izNmzfP3dEAmHTmzBl17txZCxYsqPb9WbNmae7cuVq0aJE+/vhj+fv7KykpSefOnXNx0muDxWKRJJWXl7s5ScN39uxZSZKXl1eNxxo2F9X5iYmJSkhI0Pz58yVJVqtVrVu31sSJEzV16lRXRADgAidOnFCLFi2Ul5en2267zd1xANSDsrIydevWTQsXLtT06dPVpUsX5eTkuDsWgDqYOnWqPvjgA23ZssXdUQDUszvvvFNhYWH6y1/+Yh+755575Ofnp5dfftmNyQDUhWEYWrlypYYOHSrpx5WlERER+tOf/qTJkydLkk6fPq2wsDDl5uZqxIgRbkzrHjabTUVFRaqoqFBERIQ8PNgN2CybzaazZ8+qpKREwcHBCg8Pr/EcTxfkUnl5uXbt2qX09HT7mIeHhwYMGKBt27a5IgIAFzl9+rQkKSQkxM1JANSX8ePH61e/+pUGDBig6dOnuzsOgHqwatUqJSUl6b777lNeXp5atWqlcePGacyYMe6OBqCOevbsqSVLlujQoUNq06aN9u7dq61bt2rOnDnujgagHhUUFOjrr7/WgAED7GNBQUFKTEzUtm3brsty2TAMhYeHq6CgQF9++aW74zRowcHBatmyZa2OdUm5/N///ldVVVUKCwtzGA8LC9Pnn3/uiggAXMBqtWrSpEnq1auXOnTo4O44AOrBq6++qt27d2vHjh3ujgKgHh09elQvvvii0tLS9MQTT2jHjh1KTU2Vt7e3UlJS3B0PQB1MnTpVpaWluvnmm2WxWFRVVaVnnnlGI0eOdHc0APXo66+/lqRqu7YL712PvL29FRcXx9YYdeDl5WXfYqQ2XFIuA7g+jB8/Xvv27dPWrVvdHQVAPSguLtYf//hHbdiwQb6+vu6OA6AeWa1Wde/eXTNmzJAkde3aVfv27dOiRYsol4EG7rXXXtMrr7yi5cuXq3379srPz9ekSZMUERHB/Q3guuDh4cH/fnEhl2w+csMNN8hiseibb75xGP/mm29qvcQawLVtwoQJeuedd/T+++/rxhtvdHccAPVg165dKikpUbdu3eTp6SlPT0/l5eVp7ty58vT0VFVVlbsjArhK4eHhuuWWWxzG2rVrp6KiIjclAlBfpkyZoqlTp2rEiBHq2LGjfvOb3+ixxx7TzJkz3R0NQD260KfRtcHdXFIue3t7Kz4+Xhs3brSPWa1Wbdy4UT169HBFBABOYrPZNGHCBK1cuVL//ve/FRMT4+5IAOpJ//799emnnyo/P9/+6N69u0aOHKn8/HxT/1cpANeWXr166eDBgw5jhw4dUlRUlJsSAagvZ8+eveRHrCwWi6xWq5sSAXCGmJgYtWzZ0qFrKy0t1ccff0zXBpdy2bYYaWlpSklJUffu3XXrrbcqJydHZ86c0ejRo10VAYATjB8/XsuXL9fbb7+tgIAA+95OQUFB8vPzc3M6AHUREBBwyf7p/v7+at68OfuqAw3cY489pp49e2rGjBkaPny4tm/friVLlmjJkiXujgagjoYMGaJnnnlGkZGRat++vfbs2aM5c+bo4Ycfdnc0ACaVlZXpiy++sL8uKChQfn6+QkJCFBkZqUmTJmn69OmKi4tTTEyMnnrqKUVERGjo0KHuC43rjmGz2Wyummz+/PmaPXu2vv76a3Xp0kVz585VYmKiq6YH4ASGYVQ7vmzZMo0aNcq1YQA4Xd++fdWlSxfl5OS4OwqAOnrnnXeUnp6uw4cPKyYmRmlpaRozZoy7YwGoo++//15PPfWUVq5cqZKSEkVEROiBBx5QRkaGvL293R0PgAmbNm1Sv379LhlPSUlRbm6ubDabMjMztWTJEp06dUq9e/fWwoUL1aZNGzekxfXKpeUyAAAAAAAAAKBxcMmeywAAAAAAAACAxoVyGQAAAAAAAABgGuUyAAAAAAAAAMA0ymUAAAAAAAAAgGmUywAAAAAAAAAA0yiXAQAAAAAAAACmUS4DAAAAAAAAAEyjXAYAAAAAAAAAmEa5DAAAAAAAAAAwjXIZAAAAAAAAAGAa5TIAAAAAAAAAwLT/B0QiwpYKlPS/AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1600x50 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TSTensor(samples:8, vars:3, len:5000, device=cpu, dtype=torch.float32)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = np.zeros((10, 3, 5000)) \n",
    "y = np.random.randint(0,2,X.shape[0])\n",
    "splits = get_splits(y)\n",
    "dls = get_ts_dls(X, y, splits=splits)\n",
    "xb, yb = dls.train.one_batch()\n",
    "xb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you try to use SequencerPlus, it's likely you'll get an 'out-of-memory' error.\n",
    "\n",
    "To avoid this you can subsample the sequence reducing the input's length. This can be done in multiple ways. Here are a few examples: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 99])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Separable convolution (to avoid mixing channels)\n",
    "feature_extractor = Conv1d(xb.shape[1], xb.shape[1], ks=100, stride=50, padding=0, groups=xb.shape[1]).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convolution (if you want to mix channels or change number of channels)\n",
    "feature_extractor=MultiConv1d(xb.shape[1], 64, kss=[1,3,5,7,9], keep_original=True).to(default_device())\n",
    "test_eq(feature_extractor.to(xb.device)(xb).shape, (xb.shape[0], 64, xb.shape[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MaxPool\n",
    "feature_extractor = nn.Sequential(Pad1d((0, 50), 0), nn.MaxPool1d(kernel_size=100, stride=50)).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# AvgPool\n",
    "feature_extractor = nn.Sequential(Pad1d((0, 50), 0), nn.AvgPool1d(kernel_size=100, stride=50)).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once you decide what type of transform you want to apply, you just need to pass the layer as the feature_extractor attribute:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 1000\n",
    "c_out = 2\n",
    "d_model = 128\n",
    "\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "feature_extractor = partial(Conv1d, ks=5, stride=3, padding=0, groups=xb.shape[1])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, d_model=d_model, feature_extractor=feature_extractor)\n",
    "test_eq(model.to(xb.device)(xb).shape, (bs, c_out))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsai.utils import alphabet, ALPHABET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W NNPACK.cpp:53] Could not initialize NNPACK! Reason: Unsupported hardware.\n"
     ]
    }
   ],
   "source": [
    "a = alphabet[np.random.randint(0,3,40)]\n",
    "b = ALPHABET[np.random.randint(6,10,40)]\n",
    "c = np.random.rand(40).reshape(4,1,10)\n",
    "map_a = {k:v for v,k in enumerate(np.unique(a))}\n",
    "map_b = {k:v for v,k in enumerate(np.unique(b))}\n",
    "n_cat_embeds = [len(m.keys()) for m in [map_a, map_b]]\n",
    "szs = [emb_sz_rule(n) for n in n_cat_embeds]\n",
    "a = np.asarray(a.map(map_a)).reshape(4,1,10)\n",
    "b = np.asarray(b.map(map_b)).reshape(4,1,10)\n",
    "inp = torch.from_numpy(np.concatenate((c,a,b), 1)).float()\n",
    "feature_extractor = partial(Conv1d, ks=3, padding='same')\n",
    "model = TSSequencerPlus(3, 2, 10, d_model=64, cat_pos=[1,2], feature_extractor=feature_extractor)\n",
    "test_eq(model(inp).shape, (4,2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequence Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sometimes you have a samples with a very long sequence length. In those cases you may want to reduce it's length before passing it to the transformer. To do that you may just pass a token_size like in this example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128, 168])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.rand(8, 2, 10080)\n",
    "SeqTokenizer(2, 128, 60)(t).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 5])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.rand(8, 2, 10080)\n",
    "model = TSSequencerPlus(2, 5, 10080, d_model=64, token_size=60)\n",
    "model(t).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": "IPython.notebook.save_checkpoint();",
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/nacho/notebooks/tsai/nbs/069_models.TSSequencerPlus.ipynb saved at 2023-03-26 16:01:39\n",
      "Correct notebook to script conversion! 😃\n",
      "Sunday 26/03/23 16:01:41 CEST\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
       "                    <source src=\"data:audio/wav;base64,UklGRvQHAABXQVZFZm10IBAAAAABAAEAECcAACBOAAACABAAZGF0YdAHAAAAAPF/iPh/gOoOon6w6ayCoR2ZeyfbjobxK+F2Hs0XjKc5i3DGvzaTlEaraE+zz5uLUl9f46fHpWJdxVSrnfmw8mYEScqUP70cb0Q8X41uysJ1si6Eh1jYzXp9IE2DzOYsftYRyoCY9dJ/8QICgIcEun8D9PmAaBPlfT7lq4MFIlh61tYPiCswIHX+yBaOqT1QbuW7qpVQSv9lu6+xnvRVSlyopAypbGBTUdSalrSTaUBFYpInwUpxOzhti5TOdndyKhCGrdwAfBUcXIJB69p+Vw1egB76+n9q/h6ADglbf4LvnIHfF/981ODThF4m8HiS0riJVjQ6c+/EOZCYQfJrGrhBmPVNMmNArLKhQlkXWYqhbaxXY8ZNHphLuBJsZUEckCTFVHMgNKGJytIDeSUmw4QN4Qx9pReTgb3vYX/TCBuApf75f+P5Y4CRDdN+B+tngk8c8nt03CKGqipgd13OhotwOC5x9MCAknFFcmlmtPmagFFFYOCo0qRzXMhVi57pryNmIEqJlRi8bm52PfuNM8k4dfQv+4cO12l6zCGdg3jl730uE/KAPvS+f0wEAoAsA89/XfXQgBESIn6S5luDtiC8eh/YmIfpLqt1OMp5jXg8/24MveqUNUnPZsqw0Z3yVDldnaUOqIZfXlKrm36zzWhjRhaT+r+ncHI5/otUzfd2uSt7hl/bqXtoHaCC6+mqfrAOeoDD+PJ/xf8RgLMHfH/b8GeBihZIfSXidoQSJWB52NM1iRkzz3MkxpKPbUCrbDu5d5fgTAxkSK3JoEhYD1p2omere2LZTuqYLbdWa49Cx5Dww7tyXDUnioXRkHhwJyKFvd/AfPoYy4Fl7j1/LQorgEr9/X89+0qAOAwAf13sJoL8Gkd8wt25hWIp3Heez/eKODfPcSPCzpFNRDVqf7UlmnNQKGHgqd+jgVvJVm2f265QZTpLS5byur1tpT6ajvrHq3Q2MXWIxtUCehoj8YMk5LB9hRQegeTypn+nBQWA0QHgf7f2q4C5EFt+5ucOg2YfHXtq2SSHpS0ydnTL4IxFO6pvNb4ulBdInWfcsfSc7VMmXpSmE6eeXmZThJxpsgRohEfOk86+AHCoOpOMFsx1dv8s6oYT2k17uR7ngpXod34IEJqAaPfnfyABCIBZBpl/NPI2gTQVjX134x2ExSPMeR7VtYjZMWJ0W8ftjkA/YW1durCWykvjZFKu4p9LVwVbZKNkqpxh6U+6mRC2mGq2Q3SRvsIgcpc2sIpD0Bp4uiiFhW3ecXxOGgaCDe0Vf4cLPoDv+/5/mfw1gN4KKX+17emBqBmYfBHfVYUZKFR44NBtiv41bHJUwx+RJkP1apu2VJlkTwli4qrwoo1ax1dToNCtemRSTBGXz7kJbdM/PY/Dxht0dTLziH7Ul3loJEiE0uJsfdsVTYGL8Yt/AgcMgHYA7X8S+IqAYA+QfjzpxIIVHnp7tdqzhmAstXaxzEqMETpScGC/dJP3Rmdo8LIZnOVSEF+Opxumsl1sVF+dVrE5Z6NIiZSkvVdv2zsqjdnK8HVDLlyHyNjuegogM4NA5z9+YRG9gA722H97AgOA/gSyf43zCIHdE899yuTIg3ciNXpm1jmImTDwdJPITI4RPhRugbvslbFKt2Vfr/6eTFb4W1WkY6m6YPdQjJr2tNZp3EQlko7BgXHRNz2LAc+gdwMq7IUf3R58ohtFgrbr6n7hDFWAlPr8f/T9I4CECU9/De+vgVQY5nxh4POEzybJeCTS5YnCNAZzhsRzkP1Bsmu4t4aYU07nYuerA6KWWcJYO6HHrKJjaE3Zl624UWz/QOOPjcWHc7QzdIk40yl5tCWjhIDhJX0xF4CBMvBsf10IF4Ac//Z/bPlsgAcOwn6S6n6CwxzUewLcRoYaKzV38M23i9o493CNwL6S1UUuaQe0QpvbUfdfiqglpcRccFU+nkWwambASUiVfLyqbg49xY2eyWh1hy/Sh37XjHpaIYKD7OUEfrgS5IC09MV/1gMBgKMDyH/n9N6AhhINfh7mdoMoIZt6r9fAh1cvfHXNya6N4DzDbqi8K5WWSYlmbbAdnkpV6FxJpWSo1V8DUmGb3rMRaQBG2JJgwN9wCDnNi8HNI3dKK1aG0dvHe/UciIJf6rt+Og5wgDn59X9P/xWAKQhxf2XweYH+FjB9suGVhIMlOnlo02GJhTOdc7vFyo/TQGxs2Li7lz9NwmPurBihnVi7WSWiwKvGYntOpJiOt5drKUKMkFnE8HLxNPmJ9NG4eP8mAYUv4Np8hhi3gdruSX+3CSWAwP38f8f6UoCuDPF+6Os8gnAbKnxQ3d2F0imydzDPKIuiN5lxu8EKkrFE82kftW2az1DbYImpMqTUW3FWIJ83r5hl2koJlla7+m0+PmSOZcjcdMgwS4g11iZ6qCLUg5jkxn0QFA6BWvOvfzEFBIBHAtp/Qfa3gC4RSH5y5yeD2B/8evnYS4cULgR2CMsUja47cG/QvW6UeEhXZ3+xP51GVNVdP6Zpp+1eDFM5nMeySWghR4+TNL85cD46YIyCzKJ2kCzEhoTabXtGHs+CCemJfpMPjoDe9+t/qQALgM8Gj3++8UaBqRV2fQTjO4Q3JKd5r9TgiEYyMHTxxiWPpz8jbfq585YpTJpk960xoKFXsVoTo7yq6GGMTw==\" type=\"audio/wav\" />\n",
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#|eval: false\n",
    "#|hide\n",
    "from tsai.export import get_nb_name; nb_name = get_nb_name(locals())\n",
    "from tsai.imports import create_scripts; create_scripts(nb_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
